import re
from datetime import timedelta, datetime
from urllib.parse import urljoin

import scrapy
from scrapy import Request
from scrapy.utils.project import get_project_settings

from wuliu_news.api.addnews import AddNews
from wuliu_news.api.dicts.zhishujiedu_dict import zhishujiedu_catid_dict
from wuliu_news.items import WuliuJiageZhishuItem


class SteelhomeSpider(scrapy.Spider):
    name = 'steelhome'
    allowed_domains = ['steelhome.cn']
    start_urls = []

    jgzsTypes = {
        "https://www.steelhome.cn/shpi/shpi_price_map_ll.php": "原料",
        'https://www.steelhome.cn/shpi/index.php': "钢铁",
        "https://www.steelhome.cn/shpi/shpi_qpz.php": "钢铁",
        "https://www.steelhome.cn/shpi/shpi_price_map_gc.php": "钢铁",
        "https://www.steelhome.cn/gjshpi/index.php": "钢铁",
        "https://www.steelhome.cn/shpi-l/index.php": "原料",
        "https://www.steelhome.cn/tksshpi/shpi_tkspz.php": "原料",
        "https://www.steelhome.cn/shpi_zh/shpi_qzhtb.php?class=gp": "原料",
        "https://www.steelhome.cn/shpi_zh/shpi_qzhtb.php?class=fg": "原料",
        "https://www.steelhome.cn/shpi_zh/shpi_qzhtb.php?class=thj": "原料",
        "https://www.steelhome.cn/shpi_zh/shpi_qzhtb.php?class=ys": "有色",
        "https://www.steelhome.cn/shpi_zh/shpi_qzhtb.php?class=sn": "水泥",
        "https://www.steelhome.cn/shpi_zh/shpi_qzhtb.php?class=jt": "煤焦",
        "https://www.steelhome.cn/shpicoal/shpicoal_mj.php?zhishu=coal": "煤焦",
    }

    zsjdTypes = {
        "https://news.steelhome.cn/gc/col-076/": "钢铁",
        "https://news.steelhome.cn/ll/col-076/": "原料",
        "https://news.steelhome.cn/mj/col-076/": "煤焦",
        "https://news.steelhome.cn/thj/col-076/": "原料",
        "https://news.steelhome.cn/ll/col-076/var-c13/": "原料",
        "https://news.steelhome.cn/tg/col-076/": "钢铁",
    }

    moduleid = 42

    settings = get_project_settings()
    FROM_DOMAIN = settings.get("SOURCE_DOMAIN")
    limit_day = settings.get("TODAY") - timedelta(days=settings.get("CRAWL_CYCLE", default=1) - 1)

    def start_requests(self):
        for url_jgzs, jgzsType in self.jgzsTypes.items():
            jgzs_headers = {
                "Referer": url_jgzs,
                "Host": "www.steelhome.cn",
            }
            yield Request(url=url_jgzs, headers=jgzs_headers, callback=self.parse, dont_filter=True,
                          # 例：对水泥
                          meta={'jgzsType': jgzsType}
                          )

        for url_zsjd, zsjdType in self.zsjdTypes.items():
            zsjd_headers = {
                "Referer": url_zsjd,
                "Host": "news.steelhome.cn",
            }
            yield Request(url=url_zsjd, headers=zsjd_headers, callback=self.parse_zsjd_list, dont_filter=True,
                          meta={'zsjdType': zsjdType}
                          )

    def parse(self, response, **kwargs):
        """
        :param response:
        :param kwargs:
        :return:
        """
        jgzsType = response.meta['jgzsType']
        catid = zhishujiedu_catid_dict.get(jgzsType, 47354)

        base_pic_url = response.url.strip().rsplit("/", maxsplit=1)[0]
        base_pic_url_tail = "/" + base_pic_url.rsplit("/", maxsplit=1)[1] + "/"

        titles = response.css(".redtit::text").getall()

        jgzspicItem = WuliuJiageZhishuItem(cat_id=catid)
        jgzspicItem.title = titles[0]
        jgzspicItem.introduce = jgzspicItem.title

        jgzspicItem.copyfrom = "www.steelhome.cn"
        jgzspicItem.fromurl = response.url
        jgzspicItem.addtime = str(datetime.now().date())

        # 走势图
        src = response.css("img#shpi_img1::attr(src)").get()
        if src:
            if "steelhome.cn" in src:
                src = src.split("steelhome.cn", maxsplit=1)[1]

            pic_src = src.lstrip('./')
            pic_url = urljoin(base_pic_url, pic_src)
            pic_url = pic_url if base_pic_url_tail in pic_url else f"{base_pic_url}/{pic_src}"

            upload_url = AddNews.upload_pic(pic_url)

            content = f"""
                    <img src="{upload_url or pic_url}" alt="{jgzspicItem.title}">
                    """
            jgzspicItem.content = content

            jgzspicItem.set_defaults(skip_time=True)
            values = jgzspicItem.values
            yield values
        else:
            # 有的走势图嵌入iframe中，不能同时取到，需要另外发起请求
            image_page_src = response.css("#shpi_img1::attr(src)").get()
            if "steelhome.cn" in image_page_src:
                image_page_src = image_page_src.split("steelhome.cn", maxsplit=1)[1]

            image_page_src = image_page_src.lstrip('./')
            image_page_url = urljoin(base_pic_url, image_page_src)
            image_page_url = image_page_url
            headers = {
                "Referer": response.url,
                "Host": "www.steelhome.cn",
            }
            yield Request(url=image_page_url, headers=headers, callback=self.parse_image_page, dont_filter=False,
                          meta={'item': jgzspicItem})

        # 价格表格
        if len(titles) > 2:
            jgzstableItem = WuliuJiageZhishuItem(cat_id=catid)
            jgzstableItem.title = titles[1]
            jgzstableItem.introduce = jgzstableItem.title
            jgzstableItem.addtime = str(datetime.now().date())
            jgzstableItem.fromurl = response.url
            jgzstableItem.copyfrom = "www.steelhome.cn"

            content = response.css("table.table").get()
            content = re.sub("<img src=[^>]+?>", "", content)
            jgzstableItem.content = content

            jgzstableItem.set_defaults(skip_time=True)
            values = jgzstableItem.values
            yield values

    def parse_image_page(self, response, **kwargs):
        """"""
        jgzspicItem: WuliuJiageZhishuItem = response.meta['item']
        base_pic_url = jgzspicItem.fromurl.rsplit("/", maxsplit=1)[0]
        base_pic_url_tail = "/" + base_pic_url.rsplit("/", maxsplit=1)[1] + "/"

        image_src = response.css("body>img::attr(src)").get()

        image_src = image_src.lstrip('./')
        image_url = urljoin(base_pic_url, image_src)
        image_url = image_url

        upload_url = AddNews.upload_pic(remote_url=image_url)

        content = f"""
        <img src="{upload_url or image_url}" alt="{jgzspicItem.title}">
        """
        jgzspicItem.content = content

        jgzspicItem.set_defaults(skip_time=True)
        values = jgzspicItem.values
        yield values

    def parse_zsjd_list(self, response, **kwargs):
        """
        :param response:
        :param kwargs:
        :return:
        """
        zsjdType = response.meta['zsjdType']
        catid = zhishujiedu_catid_dict.get(zsjdType, 47354)

        article_divs = response.css(".c_middle>form>div")
        for div_elem in article_divs:
            date_str = div_elem.css(".newslist_date::text").get(default="").strip()
            if not date_str:
                continue
            # 1月3日 or 21/12/31
            # 去年的就跳过吧
            if "月" not in date_str or "21" == date_str[:2] or "2021" == date_str[:4]:
                continue
            month = date_str.split("月")[0]
            day = date_str.split("月")[1].strip("日")
            date = datetime(year=datetime.now().year, month=int(month), day=int(day))

            if date.date() < self.limit_day:  # - timedelta(days=1)
                break

            title = div_elem.css(".newslist_red>a::attr(title)").get()
            if not title:
                title = div_elem.css(".newslist_red>a::text").get()
            url = href = div_elem.css(".newslist_red>a::attr(href)").get()

            zsjdItem = WuliuJiageZhishuItem(cat_id=catid)
            zsjdItem.title = title
            zsjdItem.addtime = str(date)
            zsjdItem.introduce = zsjdItem.title
            zsjdItem.fromurl = url
            zsjdItem.copyfrom = "www.steelhome.cn"

            headers = {
                "Referer": response.url
            }
            yield Request(url=url, headers=headers, callback=self.parse_zsjd_detail, dont_filter=True,
                          meta={'item': zsjdItem}
                          )

    def parse_zsjd_detail(self, response, **kwargs):
        """
        :param response:
        :param kwargs:
        :return:
        """
        zsjdItem: WuliuJiageZhishuItem = response.meta['item']
        if "一周综述" in zsjdItem.title:
            content = response.css("#article_main").get()
        else:
            contents = response.css("#sth_content>p").getall()
            if len(contents) > 1:
                last_cont = contents[-1]
                if len(last_cont.encode()) < 70 and "简述" in last_cont[:10] and "……" in last_cont[-10:]:
                    contents = contents[:-1]

            content = ''.join(contents)

        execpt_info = "此信息为当天免费信息，请登录后查看。"

        p_texts = ''.join(t.strip() for t in response.css("#sth_content>p ::text").getall())
        if not p_texts.replace(execpt_info, "").strip():
            return

        content = content.replace(execpt_info, "")
        if not content.strip():
            return

        zsjdItem.content = content

        zsjdItem.set_defaults(skip_time=True)
        values = zsjdItem.values
        yield values
